package com.strat.searcher.analisys.recognize;

import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.Validate;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.strat.searcher.analisys.tokenize.Token;
import com.strat.searcher.analisys.tokenize.TokenType;
import com.strat.searcher.analisys.tokenize.TokenUtil;
import com.strat.searcher.exceptions.TextAnalisysException;

/**
 * A recognizer that recognizes common stop words. Stopwords should be
 * initialized by non-default constructor.
 * 
 * @author Oleksandr Stratiyevskyy
 * @version $Revision$
 * 
 */
public class StopwordRecognizer implements Recognizable {

    private final Logger logger = LoggerFactory.getLogger(this.getClass());

    private Set<String> stopwords = new TreeSet<String>();

    private File file = new File("");

    public StopwordRecognizer(File file) {
        super();
        this.file = file;

    }

    public void init() {
        Validate.isTrue(this.file.exists());
        List<String> stopwordFileLines = new LinkedList<String>();
        try {
            stopwordFileLines = FileUtils.readLines(file, "UTF8");
        } catch (IOException e) {
            logger.error("error", e);
            throw new TextAnalisysException(e);
        }
        this.stopwords.addAll(stopwordFileLines);

    }

    public List<Token> recognize(List<Token> tokens) {
        List<Token> recognizedTokens = new LinkedList<Token>();
        for (Token token : tokens) {
            if (token.getType() == TokenType.WORD) {
                if (stopwords.contains(StringUtils.lowerCase(token.getText()))) {
                    token.setType(TokenType.STOP_WORD);

                }
            }
            recognizedTokens.add(token);
        }
        logger.info("left "
                + TokenUtil.filtredTokens(recognizedTokens, TokenType.WORD)
                        .size());
        return recognizedTokens;
    }

    public Set<String> getStopwords() {
        return stopwords;
    }

    public File getFile() {
        return file;
    }

}
